<!DOCTYPE HTML>
<HTML>
	<HEAD>
		<TITLE>Indexing</TITLE>
		<script type="text/javascript" src="jquery.js"></script>
		<script type="text/javascript" src="jquery.filedrop.js"></script>
		<script type="text/javascript" src="porterStemmer.js"></script>
		<style type="text/css">
			html, body, p{
				margin: 0;
				padding: 0;
			}
			#dropbox{
				width: 100%;
				height: 200px;
				line-height: 200px;
				text-align: center;
				background-color: #ffffcc;
				font-weight: bold;
			}
		</style>
	</HEAD>
	<BODY>
		<div id="dropbox">
			<p>Drop text-file(s) here</p>
		</div>
		<input type="text" id="query"></input><input type="button" id="search" value="Search"></input>
		<pre id="output"></pre>
		<pre id="debugLeft" style="float: left; width: 48%;">
			
		</pre>
		<pre id="debugRight" style="float: right; width: 48%;">
			
		</pre>
		<script type="text/javascript">
			$(function(){
			
				var filesIndex = {};
				var wordsIndex = {};
			
				var readTextFiles = function(files){
					readNextTextFile(files, 0);
				};
				
				var readNextTextFile = function(files, i){
					if(i < files.length){
						readTextFile(files[i], function(){
							readNextTextFile(files, i+1);
						});
					} else {
						calcTf();
						calcIdf();
						calcWeight();
						$("#debugLeft").html("filesIndex: " + JSON.stringify(filesIndex, null, "\t"));
						$("#debugRight").html("wordsIndex: " + JSON.stringify(wordsIndex, null, "\t"));
					}
				};
				
				var readTextFile = function(file, onFinish){
					if(file.type != "text/plain"){
						console.error("Unsupported file type: \"" + file.type + "\" at file " + file.name, file);
						onFinish();
					} else {
						console.log("Reading file " + file.name, file);
						var reader = new FileReader();
						reader.onload = function(event){
							var contents = event.target.result;
							addToIndex(splitText(contents), file.name, onFinish);
						};
						reader.readAsText(file);
					}
				};
				
				var splitText = function(text){
					var split = text.toLowerCase().split(/[^a-z]/g);
					var filtered = {};
					var max = 0;
					for(var i in split){
						var word = stemmer(split[i].trim());
						if(word != ""){
							if(filtered[word]){
								filtered[word].count += 1;
							} else {
								filtered[word] = {count: 1};
							}
							if(filtered[word].count > max) max = filtered[word].count;
						}
					}
					return {max: max, words: filtered};
				};
				
				var addToIndex = function(fileObj, filename, onFinish){
					filesIndex[filename] = fileObj;
										
					for(var i in fileObj.words){
						if(wordsIndex[i]){
							wordsIndex[i].files.push(filename);
						} else {
							wordsIndex[i] = {files:[filename]};
						}
					}
					
					onFinish();
				};
				
				var calcTf = function(){
					for(var i in filesIndex){
						for(var j in filesIndex[i].words){
							filesIndex[i].words[j].tf = filesIndex[i].words[j].count / filesIndex[i].max;
						}
					}
				}
				
				var calcIdf = function(){
					var N = Object.keys(filesIndex).length;
					for(var i in wordsIndex){
						wordsIndex[i].idf = Math.log(N*1.0 / wordsIndex[i].files.length)/Math.log(2);
					}
				};
				
				var calcWeight = function(){
					for(var i in filesIndex){
						for(var j in filesIndex[i].words){
							filesIndex[i].words[j].w = filesIndex[i].words[j].tf * wordsIndex[j].idf;
						}
						
						var documentLength = 0;
						for(var j in filesIndex[i].words){
							documentLength += filesIndex[i].words[j].w * filesIndex[i].words[j].w;
						}
						documentLength = Math.sqrt(documentLength);
					
						filesIndex[i].length = documentLength;
					}
				};
				
				var dragEnterLeave = function(e) {
					e.stopPropagation();
					e.preventDefault();
				};
				
				$("#dropbox").on("dragenter", dragEnterLeave);
				$("#dropbox").on("dragleave", dragEnterLeave);
				$("#dropbox").on("dragover", dragEnterLeave);
				$("#dropbox").on("drop", function(e){
					readTextFiles(e.dataTransfer.files);
					e.stopPropagation();
					e.preventDefault();
				});
				
				$("#search").click(function(){
					var query = $("#query").val();
					var split = splitText(query);
										
					queryLoop: for(var i in split.words){
						split.words[i].tf = split.words[i].count / split.max;
						for(var j in wordsIndex){
							if(j == i){
								split.words[i].w = split.words[i].tf * wordsIndex[j].idf;
								continue queryLoop;
							}
						}
						split.words[i].w = 0;
					}
					//$("#output").html(JSON.stringify(split, null, "\t"));
					
					var queryVector = [];
					var documentVectors = {};
					
					for(var j in split.words){
						queryVector.push(split.words[j].w);
					}
					
					for(var i in filesIndex){
						wordsLoop: for(var j in split.words){
							for(var k in filesIndex[i].words){
								if(j == k){
									if(!documentVectors[i]) documentVectors[i] = [];
									documentVectors[i].push(filesIndex[i].words[k].w);
									continue wordsLoop;
								}
							}
							if(!documentVectors[i]) documentVectors[i] = [];
							documentVectors[i].push(0);
						}
					}
					
					//$("#output").html(JSON.stringify(queryVector, null, "\t") + "\n\n" + JSON.stringify(documentVectors, null, "\t"));
					
					var numerator = 0;
					
					var queryLength = 0;
					for(var i in queryVector){
						queryLength += queryVector[i] * queryVector[i];
					}
					queryLength = Math.sqrt(queryLength);
					
					
					var documentRank = [];
					
					for(var i in documentVectors){
						var numerator = 0;
						for(var j in documentVectors[i]){
							numerator += documentVectors[i][j] * queryVector[j];
						}
						
						var denominator = queryLength * filesIndex[i].length;
						
						var rank = numerator/denominator
						if(denominator == 0) rank = 0;
						
						documentRank.push({
							file: i,
							rank: rank,
						});
					}
					
					documentRank.sort(function(A,B){
						if(A.rank < B.rank) return 1;
						return -1;
					});
					
					$("#output").html(JSON.stringify(documentRank, null, "\t"));
					
				});
			
			});
		</script>
	</BODY>
</HTML>